#Author:SSun
# -*- codeing = utf-8 -*-
# @Time :2020/8/27 19:37
# @Author:孙孙
# @Site : 
# @File : spider_joke.py
# @Sortware: PyCharm

import requests
import logging
from lxml import etree
import pymysql
from pymysql import err

from datetime import datetime
import time
import random


from user_agent import User_agent
from config import Config


logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s - %(levelname)s: %(message)s')



class Joke_spider():

    def __init__(self):
        self.url = 'https://www.qiushibaike.com/text/'
        self.ua = User_agent()
        self.time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        self.conn = pymysql.connect(
            host = Config.HOST,
            user = Config.USERNAME,
            password = Config.PASSWORD,
            port = Config.PORT,
            charset = 'utf8mb4',
            db = Config.DBNAME
        )


    def urls(self):

        nums = 13
        for num in range(nums):
            url = self.url + 'page/' + str(num+1) + '/'
            yield url


#获取网页
    def ask_url(self):

        for url in self.urls():
            headers = self.ua.user_agent_header()
            resp = requests.get(url, headers=headers)
            resp.encoding = resp.apparent_encoding
            html = etree.HTML(resp.content)
            logging.info('获取网页'+url)
            yield html



#解析网页
    def parse_html(self, html):
        html = html
        contents = html.xpath('//div[@class="article block untagged mb15 typs_hot"]//div[@class="content"]/span')
        for content in contents:
            # 获取span段落下所有字符
            text = content.xpath('string(.)').strip()
            # 去除\n
            text = "".join([i.replace("\n","") for i in text])
            text = "".join(text.split())
            yield text



#将数据导入数据库

    def write_db(self, html):

        try:
            cons = self.parse_html(html)
            for con in cons:
                conn = self.conn
                cursor = conn.cursor()
                body = con
                print(con)
                print("="*30)
                time = self.time
                print(time)
                body_header = con[0:10]
                sql = """
                          insert into joke_content values(null,'%s','%s','%s');
                      """ %(body, time, body_header)

                cursor.execute(sql)
                conn.commit()
                cursor.close()
        except err.IntegrityError:
            print("文章重复")




        # except:
        #     logging.info("文章重复")





    def readdb(self):

        cursor = self.conn.cursor()
        sql = """
            select * from joke_content
        """
        cursor.execute(sql)
        books = cursor.fetchall()
        print(books)
        cursor.close()




# if __name__ == '__main__':
#     joke = Joke_spider()
#     html = joke.ask_url()
#     texts = joke.parse_html(html)
#     joke.write_db(html)
